#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
PErsonal Python Enhancements library (general purpose and data-mining)

Copyright (C) 2010, "Fatura, Karel" <kfatura@deloitteCE.com>
Copyright (C) 2007-2010, "Viktor Presovsky" <vpresovsky@deloitteCE.com>
Copyright (C) 2003-2007, "Viktor Presovsky" <presovsky@gmail.com>
Copyright (C) 2009, "Michal Sagat" <msagat@deloitteCE.com>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

import datetime
import glob
import os
import platform
import re
import shutil
import string
import sys
import time

# posix 2.4, nt 2.4
OSPYVER = "%s %s" % (os.name, platform.python_version()[:-2])
#print OSPYVER

import pepe.lambdas as lamb
#import pepe.p_dm.dama as dama
import pepe.p_dm.sdb as s
#import pepe.p_sql.pg as pg
import pepe.p_txt.bigtxt as bt
import pepe.p_txt.txt_columns as tc
import pepe.p_utils.srchr as srchr
import pepe.p_utils.txtr as p_txtr
import pepe.p_utils.uid_uuid as uuid

zf9 = lambda x: string.zfill(str(x), 9)
x3x = lambda x: "*%s.%s.%s*" % (zf9(x)[:3], zf9(x)[3:6], zf9(x)[6:9])
# 2683099 -> *002.683.099*

fw_slash = lambda x: re.sub(r'\\', '/', x)

#print lamb.dtnow() # datetime stamp
timenow = lambda : time.strftime ("%Y%m%dT%H%M%S", time.localtime())
#print timenow()
dtnow = lambda : re.sub('[-. :]', '', str(datetime.datetime.now())[:])
d_tnow = lambda : time.strftime ("%Y%m%d_%H%M%S", time.localtime())

def file_hash(ifile, type='sha256'):
    ''' calculate checksum for file
    '''
    import pepe.p_filesystem.dfstats as dfs
    
    rd = dfs.dfstats(ifile, options=[type], pprint=False)
    for f in sort_case_insensitive(rd.keys()):
        x = "%s *%s\n" % (rd[f][type], rd[f]['file_name'])
        return x

def strip_txt_columns(txt, cd=''):
    ''' '''
    #line = re.sub('\n', '', txt)
    cols = string.split(line, cd)
    bare_cols = [string.strip(c) for c in cols]
    return string.join(bare_cols, cd)

def sort_case_insensitive(stringList):
    ''' 
    http://code.activestate.com/recipes/170242/
    
    The default compare function used when sorting a list of strings uses the ordinal value 
    of the string characters for comparison. This results in the typical ASCII sort result 
    of 'B' < 'a'. If a user is going to see the list, it is generally better to do a case-insensitive comparison.
	
	Note: doesn't do locale-specific compare

    L = ['bank_ucty.txt', 'chs.dat#', 'dodav_pop.txt', 'dr_dokladu.txt', 'polozky.7z', 'uc_klice.txt', 'Dodavatele.txt', 'SR71.xls']
    
    print sorted(L)
    ['Dodavatele.txt', 'SR71.xls', 'bank_ucty.txt', 'chs.dat#', 'dodav_pop.txt', 'dr_dokladu.txt', 'polozky.7z', 'uc_klice.txt']
    
    print sort_case_insensitive(L)
    ['bank_ucty.txt', 'chs.dat#', 'dodav_pop.txt', 'Dodavatele.txt', 'dr_dokladu.txt', 'polozky.7z', 'SR71.xls', 'uc_klice.txt']
    '''
    tupleList = [(x.upper(), x) for x in stringList]
    tupleList.sort()
    return [x[1] for x in tupleList]

def findcoldelimiter_line(line):
    ''' guess what text file column delimiter is used 

    if cd == '': cd = findcoldelimiter_line(line)

    >>> findcoldelimiter_line("a|^b;^c~^d~")
    '^'
    '''
    c = 0; r = '' 
    # 2011-01-12_2117 vp added delimiter \x7F ... encountered in text files exported from SAP
    for cs in ['\",\"', ',', '\t', ';', '^', '~', '|', '\x7F', '\x00']:
        if string.count(line, cs) > c:
            c = string.count(line, cs); r = cs

    # 2011-05-31_1421 vp
    # for files with one column set some arbitrary non empty delimiter
    # otherwise checking routines will raise 'delimiter not found' error 
    #   todo:   expression below is not best option
    #           ... it will set \t even to unknown multiple columns tables
    if r == '': r = '\t'

    return r

def findcoldelimiter(line):
    ''' maintained for backward compatibility purposes 
        use findcoldelimiter_line() in new projects
    '''
    return findcoldelimiter_line(line)

def findcoldelimiter_file(inputfile):
    ''' '''
    line = open(inputfile, 'r').readline()
    return findcoldelimiter_line(line)

def lol_column(L, i):
    ''' return column from list-of-lists 
        L - list of lists
        i - column index
    >>> lol_column([[1,2],[3,4]], 1)
    [2, 4]
    '''
    return [x[i] for x in L]

"""    
def oldString2number(x):

    # handle following in some utf8 files ... 65Â 000,00
    x = re.sub('\xC2\xA0','', x)
    x = re.sub('Â ','', x)

    res = 0
    x = re.sub('[+ěĚšŠčČřŘžŽýÝáÁíÍéÉóÓúÚ a-zA-Z_]', '', x)

    # dot or coma : .., or ,,.
    doc = re.sub('(-| |\d)', '', x)
    if x in ('-', '0-', '-0', '.', ','): x = ''
    if len(x) == 0:
        return 0
    else:
        if len(doc) <> 0:
            # case 1.234,45-
            if doc[-1] == ',': res = float(re.sub(',', '.', re.sub('(\.|-)', '', x)))
            # case 1,234.45-
            if doc[-1] == '.': res = float(re.sub('(,|-)', '', x))
        else:
            res = float(x)
        if string.find(x, '-') <> -1: res = res*-1
        return res
"""

def is_number(x):
    ''' check whether text is number
    
    >>> is_number('íáýsas,msmsas _')
    False
    
    >>> is_number('')
    True
    
    >>> is_number('-9991234,45')
    True

    >>> is_number('( -9991234,45 )')
    True
    '''
    o = re.sub('\x0A', '', x)
    o = re.sub('\x0D', '', o)
    o = re.sub('\x0D\x0A', '', o)
    
    o = re.sub('[ ()0-9.,-]', '', o)

    o = re.sub(' ', '', o) # NON-BRAKE space v1, \xC2\xA0
    o = re.sub('Â', '', o) # NON-BRAKE space v2, \xC3\x3F

    if len(o) == 0 and x.strip() <> '':
        return True
    elif x.strip() == '':
        return True
    else:
        return False
   
def string2number(x):
    ''' convert string to number 
    
    # 2010-12-01_1220 vp, text identified inside of function argument
    #   exception should be raised, silent dropping of text characters is not desired
    #   could indicate special cases in processed data and should be raised to attention

    # 2010-12-17_1533 vp
        string2number should behave transparently (meaning: do not perform non-obvious transformations):
        whether number is positive or negative s2n decides looking for sign -
        [] or () in string have no special meaning, and will not result in changing sign to negative
            if such behavious is desired, it should and could be placed in custom code, not in general library
            example of such custom code is below:
                def h20_string2number(s):
                one = 1.0
                if s.strip()[0] == '(':
                    one = -1.0
                return g.string2number(s) * one

    #>>> string2number(';'), string2number(':')
    #(0.0, 0.0)

    #>>> string2number('**'), string2number('***')
    #(0.0, 0.0)

    #>>> string2number('--'), string2number('---')
    #(0.0, 0.0)

    #>>> string2number('íáýsas,msmsas _'), string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě')
    #(0.0, 0.0)

    #>>> string2number('íáýsas,msmsas _'), string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě')
    #(0.0, 0.0)

    #>>> string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě'), string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě')
    #(0.0, 0.0)

    >>> string2number(''), string2number('-'), string2number('-0'), string2number('0-')
    (0.0, 0.0, 0.0, 0.0)

    >>> string2number('-0'), string2number('0-')
    (0.0, 0.0)

    >>> string2number('51.12919')
    51.129190000000001
    
    >>> string2number('9.991.234,45-'), string2number('9,991,234.45-')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45-'), string2number('-9991234.45-')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45'), string2number('-9991234.45')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('9.991.234,45-'), string2number('9,991,234.45-')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('12345'), string2number('-12,345'), string2number('-12 345')
    (12345.0, -12.345000000000001, -12345.0)

    >>> string2number('-12 345.00'),string2number('-12,34'), string2number('123 456')
    (-12345.0, -12.34, 123456.0)

    >>> string2number('234'), string2number('2-'), string2number('23,-')
    (234.0, -2.0, 23.0)

    >>> string2number('-2'), string2number(',45'), string2number('1.234.567')
    (-2.0, 0.45000000000000001, 1234567.0)

    >>> string2number('1,234,567'), string2number('-1,234,567'), string2number('-1.234.567')
    (1234567.0, -1234567.0, -1234567.0)

    >>> string2number('65 000,00') # this one has NON-BRAKE SPACE
    65000.0

    >>> string2number('1,234,567-')
    -1234567.0
    
    >>> string2number(''), string2number('-'), string2number('-0'), string2number('0-') #b
    (0.0, 0.0, 0.0, 0.0)

    >>> string2number('9.991.234,45-'), string2number('9,991,234.45-') #c
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45-'), string2number('-9991234.45-') #d
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45'), string2number('-9991234.45') #e
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number(''), string2number("-0"), string2number("0-") #g
    (0.0, 0.0, 0.0)

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number("-9991234,45-") #i
    -9991234.4499999993

    >>> string2number("-9991234.45-") #j
    -9991234.4499999993

    >>> string2number("-9991234,45") #k
    -9991234.4499999993

    >>> string2number("-9991234.45") #l
    -9991234.4499999993

    >>> string2number("12345") #m
    12345.0

    >>> string2number("-12,345") #n
    -12.345000000000001

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number('65Â 000,00')
    65000.0
    
    #>>> string2number('(USD 30,420.00-)')
    #-30420.0
    
    >>> string2number('( 30,420.00- )') # with brackets
    -30420.0

    >>> string2number('( 30,420.00 )') # with brackets
    30420.0
    '''

    # 2010-12-01_1057 vp; silent drop of text characters not desired for QA purposes
    if is_number(x) == False:
        raise 'text in function argument'

    # 1. clearing the given value
    x = x.strip()
    y = x
    x = re.sub('\xC2\xA0','', x)                                  # NON-BRAKE SPACE
    x = re.sub('Â| ','', x)                                       # handle following in some utf8 files ... 65Â 000,00
    x = re.sub('[*:;()+ěĚšŠčČřŘžŽýÝáÁíÍéÉóÓúÚ a-zA-Z_]', '', x)   # other unwanted characters
                                                                   # 2010-09-14_1155 vp added  ()
                                                                   # 2010-10-11_2259 vp added  *:;
    x = re.sub('-{2,}', '', x)                                    # 2010-10-11_1635 vp ... if more than one, its not an minus
    
    if x == '' or x == '-':
        return float(0)                                           # EXIT of: non numeric values free of . , -
    
    # 2. sign + or -?
    sign = 1
    if x[0] == '-' or re.match('\d-',x[-2:]):                    # -123 or 123- or 123,45- or -123,- but not 123,-
        sign = -1
    x = re.sub('-','',x)                                          # x does not need negative indicator any more

    # 3. number format interpretation
    # doc = dot or coma : .., or ,,.
    doc = re.sub('(-|\d)', '', x)

    if doc == '':
        return float(x) * sign                                    # EXIT of: not formated integers
    else:
        sameDelims = True                                          # are all the delimiters the same?
        prevCh = doc[0]
        for currCh in doc:
            if currCh != prevCh:
                sameDelims = False
            prevCh = currCh

    if sameDelims:
        if len(doc) == 1:                                         # 51.12919
            x = re.sub(',','.',x)
        elif re.match('\d{3}',x[-3:]):                           # is x like 123.456 - thousends delimiter or
            x = re.sub(',|\.','',x)                               # 123.45 - floating point?
        else:                                                     # i dont understand for example this 1.234.56
            raise 'could not recognize number format of: ', y
    else:
        if doc[-1] == '.':                                        # if there is a floating point, thousands delimiter
            x = re.sub(',','',x)                                  # can be deleted
        elif doc[-1] == ',':
            x = re.sub('\.','',x)                                 # for czech format '.'s needs to be dropped and the
            x = re.sub(',','.',x)                                 # last ',' is converted to floating point
        else:
            raise 'could not recognize number format of: ', y

    x = '0' + x                                                   # in the clarify proces something like '.' can
    return float(x) * sign                                       # happend - but zero on the begining soves the problem
    # EXIT of: all others here                                    # without changing the value. (+- are already droped)

from decimal import *
getcontext().prec = 12

def string2number_decimal(x):
    ''' convert string to number 
    '''

    # 1. clearing the given value
    x = x.strip()
    y = x
    x = re.sub('\xC2\xA0','', x)                                  # NON-BRAKE SPACE
    x = re.sub('Â| ','', x)                                       # handle following in some utf8 files ... 65Â 000,00
    x = re.sub('[*:;()+ěĚšŠčČřŘžŽýÝáÁíÍéÉóÓúÚ a-zA-Z_]', '', x)   # other unwanted characters
                                                                   # 2010-09-14_1155 vp added  ()
                                                                   # 2010-10-11_2259 vp added  *:;
    x = re.sub('-{2,}', '', x)                                    # 2010-10-11_1635 vp ... if more than one, its not an minus
    
    if x == '' or x == '-':
        return Decimal(0)                                           # EXIT of: non numeric values free of . , -
    
    # 2. sign + or -?
    sign = 1
    if x[0] == '-' or re.match('\d-',x[-2:]):                    # -123 or 123- or 123,45- or -123,- but not 123,-
        sign = -1
    x = re.sub('-','',x)                                          # x does not need negative indicator any more

    # 3. number format interpretation
    # doc = dot or coma : .., or ,,.
    doc = re.sub('(-|\d)', '', x)

    if doc == '':
        return Decimal(x) * sign                                    # EXIT of: not formated integers
    else:
        sameDelims = True                                          # are all the delimiters the same?
        prevCh = doc[0]
        for currCh in doc:
            if currCh != prevCh:
                sameDelims = False
            prevCh = currCh

    if sameDelims:
        if len(doc) == 1:                                         # 51.12919
            x = re.sub(',','.',x)
        elif re.match('\d{3}',x[-3:]):                           # is x like 123.456 - thousends delimiter or
            x = re.sub(',|\.','',x)                               # 123.45 - floating point?
        else:                                                     # i dont understand for example this 1.234.56
            raise 'could not recognize number format of: ', y
    else:
        if doc[-1] == '.':                                        # if there is a floating point, thousands delimiter
            x = re.sub(',','',x)                                  # can be deleted
        elif doc[-1] == ',':
            x = re.sub('\.','',x)                                 # for czech format '.'s needs to be dropped and the
            x = re.sub(',','.',x)                                 # last ',' is converted to floating point
        else:
            raise 'could not recognize number format of: ', y

    x = '0' + x                                                   # in the clarify proces something like '.' can
    return Decimal(x) * sign                                       # happend - but zero on the begining soves the problem
    # EXIT of: all others here                                    # without changing the value. (+- are already droped)


def drop_vowels(text, limit=0):
    ''' Drop vowels inside of word to make string shorter.

        Do not drop vowels on words beginning or end because this
        would significantly drop readability of those.
    
        2011-03-16 : vp, initial version

    >>> drop_vowels('Match any character in the range')
    'Mtch any chrctr in the rnge'

    >>> drop_vowels('KONTROLNI_TERMIN_TYP_PERIODY')
    'KNTRLNI_TRMN_TP_PRDY'

    >>> drop_vowels('KONTROLNI_TERMIN_TYP_PERIODY', limit=19)
    'KNI_TRN_TP_PRY'
    '''
    text = re.sub(  '(?P<x>[A-Za-z])[aeiouyAEIOUY](?P<y>[A-Za-z])'
                  , '\g<x>\g<y>'
                  , text)

    text = re.sub(  '(?P<x>[A-Za-z])[aeiouyAEIOUY](?P<y>[A-Za-z])'
                  , '\g<x>\g<y>'
                  , text)

    # compress even further if limit set
    if len(text) > limit and limit <> 0:
        text = re.sub(  '(?P<x>[A-Za-z]{2})[A-Za-z]{1,}(?P<y>[A-Za-z])'
                        , '\g<x>\g<y>'
                        , text)
        return text
    return text


if __name__== '__main__':
    """ """
    import doctest
    doctest.testmod()
